#!/usr/bin/python
#coding=utf-8
# Filename: grab.py
# CreatedBy: ChenZhengguo 2011/12/7

import ConfigParser,urllib,urllib2,string
import msvcrt, sys, os, re
import mechanize
import copy
import pickle
from lxml.html.clean import Cleaner
import time

class ParseGrab:
    '''
    Get html with logined opener,
    And analyze by mechanize
    '''
    def __init__(self,url,test_id,opener=None,charset=None,data={}):
        self._data = data
        self._html = ''
        self._error = {}
        self._url = url
        self._charset = charset
        self._serial = 0
        self._config = ConfigParser.ConfigParser()
        self._simpleHtml = ''
        self._test_id = test_id
        self._test_dir = 'swap\\' + test_id + '\\'
        self._test_filename = {
                                'FormObj':'form.obj',
                                'OriHtml':'origin.html',
                                'FormHtml':'form.html',
                                'Config':'form.conf'
                            }
        if opener :
            self._opener = opener
        else :
            self._opener = None
        self.readFormConf()

    def __grabHtml__(self):
        '''
        The mechanize has a issue: response will be closed after response.read() or ParseResponse(),
        So this func collects html.
        '''
        return False

    def grab(self):
        '''opener of urllib2'''
        if opener == None :
            self._opener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
        params = urllib.urlencode(self._data)
        req = urllib2.Request(self._url,params)
        try :    
            orginHtml = self._opener.open(req).read()
        except urllib2.URLError, e:
            if hasattr(e, 'reason'):  
                self._error['code'] = e.reason[0]
                self._error['info'] = e.reason[1]
            elif hasattr(e, 'code'):  
                self._error['code'] = e.code
                self._error['info'] = ''               
            return False
        if(self._charset == None):
            p = re.compile(r'<meta\s*.*\scharset\=(.*)\s*\"\s*\/>',re.I)
            m = p.search(originHtml[0:295])
            if m :
                self._charset = m.group(1).lower()
            else :
                self._charset = 'utf-8'        
        self._html = orginHtml.decode(self._charset).encode('utf-8')
        return True

    def grab2(self,logined=True):
        '''opener of mechanize'''
        params = urllib.urlencode(self._data)
        if logined == True:
            mechanize.install_opener(self._opener)
            request = mechanize.Request(self._url,params)
        else :
            request = mechanize.Request(self._url,params)
        try:
            response = mechanize.urlopen(request)
            response2 = mechanize.urlopen(request)
        except urllib2.URLError, e:
            if hasattr(e, 'reason'):  
                self._error['code'] = e.reason[0]
                self._error['info'] = e.reason[1]
            elif hasattr(e, 'code'):  
                self._error['code'] = e.code
                self._error['info'] = ''               
            return False
        self._html = response2.read().decode(self._charset).encode('utf-8')
        #Cleaner Just Need unicode!!
        cleaner = Cleaner(style=True, links=True, forms = False,
                                page_structure=False, safe_attrs_only=False)
        self._simpleHtml = cleaner.clean_html(self._html.decode(self._charset)).encode('utf-8')
        response2.close()
        try :
            forms = mechanize.ParseResponse(response, backwards_compat=False)
            response.close()
        except :
            self._error['code'] = 901
            self._error['info'] = 'Failed to get form'
            response.close()
            return False
        if len(forms) == 0:
            self._error['code'] = 902
            self._error['info'] = 'No form in this page'
            return False
        else:
            for form in forms :
                ####run time test   
                starttime = time.clock()
                self.analyze(form)
                print 'analyze time :'+ str(time.clock() - starttime)
                starttime = time.clock()
            return True
    
    def analyze(self,form):
        '''analyze with mechanize form funcs and add to form.conf'''
        #self.post(self.getSerialForm(form_id,self._test_filename['FormObj']))
        form_id = self.getFormId()
        self.html2file(form_id,self._html,self._test_filename['OriHtml'])
        self.html2file(form_id,self._simpleHtml,self._test_filename['FormHtml'])
        self.setSerialForm(form,form_id,self._test_filename['FormObj'])
        cnt = 0 
        self._config.add_section(form_id)
        self._config.set(form_id,'create',time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())) )
        self._config.set(form_id,'url',self._url)
        self._config.set(form_id,'action',form.action)
        self._config.set(form_id,'method',form.method)
        cnt = 0
        for ctl in form.controls :
			#XML should be better?
            if ctl.name != None :
                cnt += 1
                value = ctl.name
                if ctl.type != None :
                    value += ' ~|~ '+ ctl.type
                else :
                    value += ' ~|~ <None>'
                #some type
                value += ' ~|~ '
                if ctl.type == 'text' or ctl.type == 'hidden':
                    if ctl.value == None or string.strip(ctl.value) == '' :
                        value += '<None>'
                    else :
                        value += ctl.value
                    value += ' ~|~ <None>'
                elif ctl.type == 'radio' or ctl.type == 'select' :
                    #I don't know a radio or select control how to have two values.
                    if len(ctl.value) != 1:
                        print 1+1 
                        #do sth
                    else:
                        if ctl.value == None or string.strip(ctl.value[0]) == '' :
                            value += '<None>'
                        else:
                            value += ctl.value[0]
                        list = '['
                        for item in ctl.possible_items():
                            list += item + ','
                        list = list[0:-1] + ']'
                        value += ' ~|~ '+ list 
                elif ctl.type == 'checkbox' :
                    if len(ctl.value) == 0 or ctl.value == None :
                        value += '<None>'
                    else:
                        list = '['
                        for item in ctl.value:
                            list += item + ','
                        list = list[0:-1] + ']'
                        value += list 
                    list = '['
                    for item in ctl.possible_items():
                        list += item + ','
                    list = list[0:-1] + ']'
                    value += ' ~|~ '+ list 
                value += ' ~|~ ['
                if ctl.readonly :
                    value += 'readonly'
                if ctl.disabled:
                    value += ',disabled'
                value += ']'
                self._config.set(form_id, 'name' + str(cnt) , value)
            self.saveFormConf()
    def getInfo(self):
        return {
                'error'   : self._error,
                'url'     : self._url,
                'charset' : self._charset,
                'html'    : self._html,
                'params'  : self._data,
                'serial'  : self._serial,
                'simpleHtml' : self._simpleHtml,
                }

    def getCharset(self):
        return self._charset
    def getError(self):
        return self._error
    def getHtml(self):
        return self._html
    
    def getFormId(self):
        '''One form just call ONCE! Get id from form config file, if this id folder doesnt exist, this func will create one'''
        form_id = str(len(self._config.sections()) + 1)
        if os.path.exists(self._test_dir + form_id + '\\') == False :
            os.mkdir(self._test_dir + form_id + '\\')
        return form_id #str
    def readFormConf(self,filename=None):
        '''simple, read file to self._conf'''
        if filename == None:
            filename = self._test_filename['Config']
        self._config.read(self._test_dir + filename)
    def saveFormConf(self,filename=None):
        '''simple, save to self._conf'''
        if filename == None:
            filename = self._test_filename['Config']
        self._config.write(open(self._test_dir + filename,'wb'))

    def setSerialForm(self,form,form_id,filename=None):
        if filename == None:
            filename = elf._test_filename['FormObj']
        f = open(self._test_dir + form_id + '\\' + filename,'wb')
        pickle.dump(form, f) 
    
    def getSerialForm(self,form_id,filename=None):
        if filename == None:
            filename = elf._test_filename['FormObj']
        f = open(self._test_dir  + form_id + '\\' + filename,'r')
        obj = pickle.load(f)
        return obj
    
    def html2file(self,form_id,data,filename=None):
        if filename == None:
            filename =  self._test_filename['OriHtml']
        f = open(self._test_dir + form_id + '\\' + filename, 'wb')
        f.write(data)
        f.close()
        return True
    
    
    #In ParsePost~~~~~~
    def dataCreater(self,form):
        return 'Yeepe'
    def post(self,form):
        form.set_value('ParserTes', name="dhcp[name]")
        form.set_value('1.2.3.3', name="dhcp[subnet]")
        form.set_value('1.2.3.5', name="dhcp[range_begin]")
        form.set_value('1.2.3.6', name="dhcp[range_end]")
        form.set_value('1.2.3.7', name="dhcp[gateway]")
        form.set_value('1.2.3.9', name="dhcp[netmask]")
        help(form.set_value)
        #form.set_value('1.2.3.9', name="dhcp[eth]")
        #help(form.find_control)
        response = mechanize.urlopen(form.click())
        print response.read().decode('utf-8').encode('gbk')
           

        
        

